package com.ruoyi.task;

import com.ruoyi.common.service.CategoryService;
import com.ruoyi.common.service.IProjectItemsService;
import com.ruoyi.common.service.ProjectService;
import com.ruoyi.spider.HttpClientDownloader;
import com.ruoyi.spider.ProjectPageProcessor;
import com.ruoyi.spider.ProjectPipeline;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.scheduler.QueueScheduler;

/**
 * @author 0617
 * @version 1.0
 * @package:com.ruoyi.task
 * @date 2025/8/19 下午5:14
 * @description: TODO
 */
@Slf4j
@Component("spiderTask")
@RequiredArgsConstructor
public class SpiderTask {
    private final ProjectService projectService;
    private final IProjectItemsService projectItemsService;
    private final CategoryService categoryService;

    public void startSpider() {
        ProjectPageProcessor processor = new ProjectPageProcessor();

        Spider spider = Spider.create(processor)
                .addUrl("https://xcx.you123.top/api/v1/wx70f12f240c3854b7/getResourceList?pageNum=1")
                .setDownloader(new HttpClientDownloader())
                .addPipeline(new ProjectPipeline(projectService,projectItemsService,categoryService))
                .setScheduler(new QueueScheduler())
                .thread(3);

        spider.run();
    }
}
